test_sample_str = "Hi i'm Michael Scott"
test_sample = tokenizer.encode(test_sample_str, truncation=True, padding=True, return_tensors='pt')
test_sample = test_sample.to(device)
output = model(test_sample)
# visualize
import plotly.express as px
output = torch.softmax(output.logits, dim=1)
output = output.detach().cpu().numpy()
px.bar(x=sorted_speakers, y=output[0], title=f"Model evaluation - Test sentence: '{test_sample_str}'")
test_sample_str = "Good to see you!"
test_sample = tokenizer.encode(test_sample_str, truncation=True, padding=True, return_tensors='pt')
test_sample = test_sample.to(device)
output = model(test_sample)
# visualize
import plotly.express as px
output = torch.softmax(output.logits, dim=1)
output = output.detach().cpu().numpy()
px.bar(x=sorted_speakers, y=output[0], title=f"Model evaluation - Test sentence: '{test_sample_str}'")
# create a confusion matrix evaluating the above class predictions (normalized)
cm = confusion_matrix(val_labels, val_predictions, normalize='true')
cm_rand = confusion_matrix(val_labels, val_predictions_random, normalize='true')
# plot the confusion matrix
fig, ax = plt.subplots(1, 2, figsize=(12, 4))
sns.heatmap(cm, cmap='rocket_r', annot=True, fmt='.2f', xticklabels=sorted_speakers, yticklabels=sorted_speakers, ax=ax[0])
sns.heatmap(cm_rand, cmap='rocket_r', annot=True, fmt='.2f', xticklabels=sorted_speakers, yticklabels=sorted_speakers, ax=ax[1])
ax[0].set_ylabel('Actual')
ax[0].set_xlabel('Predicted')
ax[1].set_xlabel('Predicted')
ax[1].set_ylabel('Actual')
ax[0].set_title('Model')
ax[1].set_title('Random')
plt.show()
# visualize the first prediction's explanation
shap.plots.text(shap_values)
explained_texts
['So my looks have nothing to do with it?', 'My personal favorite is the one he made for his condo association.', 'Do not bring Shakespeare into this. How dare you play the bard card?', '[opens eyes wide in total surprise]', "Ah, well it's still very good. I bet I know someone who hasn't heard that joke... your daughter Emily. How's she doing?"]
Used texts:
'So my looks have nothing to do with it?'
'My personal favorite is the one he made for his condo association.'
'Do not bring Shakespeare into this. How dare you play the bard card?'
'Ah, well it's still very good. I bet I know someone who hasn't heard that joke... your daughter Emily. How's she doing?'